{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. 卷积样例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# input输入图像是一个四维矩阵\n",
    "# 第1维是第X个图像\n",
    "# 第2第3维是长度宽度为32*32\n",
    "# 第4维是RGB颜色通道数为3的矩阵\n",
    "input = tf.placeholder(\n",
    "    name='input',\n",
    "    shape=[None, 32, 32, 3], \n",
    "    dtype=tf.float32\n",
    ")\n",
    "\n",
    "# filter是一个四维矩阵\n",
    "# 第1第2维是过滤器尺寸width和height（5*5），\n",
    "# 第3维当前层深度（3）\n",
    "# 第4维是过滤器深度（16）\n",
    "# 初始化为正态分布的随机变量\n",
    "filter_weight = tf.get_variable(\n",
    "    name='weights', \n",
    "    shape=[5, 5, 3, 16], \n",
    "    initializer=tf.truncated_normal_initializer(stddev=0.1)\n",
    ")\n",
    "\n",
    "# biases偏置量是一个一维矩阵\n",
    "# 第1维是偏置量的深度（16）\n",
    "# 初始化为数值都是0.1的一维矩阵\n",
    "biases = tf.get_variable(\n",
    "    name='biases', \n",
    "    shape=[16], \n",
    "    initializer=tf.constant_initializer(0.1)\n",
    ")\n",
    "\n",
    "# tf.nn.conv2d的strides步长参数\n",
    "# strides步长是一个四维矩阵\n",
    "# 第1维和第4维必须为1，因为卷积层的步长只对矩阵的长宽有效\n",
    "# 第2维和第3维是长和宽的步长\n",
    "# \n",
    "# tf.nn.conv2d的padding参数只提供两种SAME以及VAILD\n",
    "# SAME表示添加0填充\n",
    "# VAILD表示不添加填充\n",
    "conv = tf.nn.conv2d(\n",
    "    input=input, \n",
    "    filter=filter_weight, \n",
    "    strides=[1,1,1,1], \n",
    "    padding='SAME'\n",
    ")\n",
    "\n",
    "# tf.nn.bias_add提供了一个方便的函数给节点添加偏置量（注意这里不能直接使用加法，因为矩阵上不同位置上的节点都需要加上同样的偏置量）\n",
    "# 例如下一层神经网络的大小为2*2，但是偏置量只有一个数（因为深度为1），而2*2矩阵的每一个项都需要加上这个偏置量。\n",
    "bias = tf.nn.bias_add(value=conv, bias=biases)\n",
    "\n",
    "# 把计算结果通过relu激活函数进行去线性化\n",
    "actived_conv = tf.nn.relu(features=bias)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 池化样例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 池化有两种常用的池化\n",
    "# tf.nn.max_pool最大值池化\n",
    "# tf.nn.avg_pool均值池化\n",
    "\n",
    "# ksize为池化层过滤器filter的尺寸，过滤器的头尾两个维度必须等于1\n",
    "# 实际中使用最多的尺寸是[1,2,2,1]或者[1,3,3,1]\n",
    "# strides为池化层过滤器步长\n",
    "pool = tf.nn.max_pool(\n",
    "    value=actived_conv, \n",
    "    ksize=[1,3,3,1], \n",
    "    strides=[1,2,2,1], \n",
    "    padding='SAME'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1  0.1\n",
      "  0.1]\n",
      "[[[[ 0.02168818  0.01285435 -0.06766106 ..., -0.07072296  0.06588759\n",
      "     0.15056656]\n",
      "   [ 0.05243896 -0.17751265  0.00194047 ..., -0.02968038  0.09518041\n",
      "     0.02195903]\n",
      "   [ 0.09532433 -0.08311255 -0.05465999 ...,  0.07416227 -0.06217621\n",
      "    -0.16228133]]\n",
      "\n",
      "  [[-0.04648024 -0.16522548 -0.0372405  ...,  0.05240355 -0.00154709\n",
      "     0.07768789]\n",
      "   [-0.19078499 -0.05449989  0.07165533 ..., -0.01213859 -0.04813964\n",
      "    -0.02143265]\n",
      "   [-0.11788438 -0.06821714 -0.14636274 ..., -0.00467992  0.10337502\n",
      "     0.02416306]]\n",
      "\n",
      "  [[ 0.10671563  0.00708553  0.08384535 ...,  0.03588026  0.07476931\n",
      "     0.05384065]\n",
      "   [-0.00400845 -0.01078473 -0.12714158 ..., -0.18333089  0.14854972\n",
      "    -0.03532339]\n",
      "   [-0.15494664 -0.01021983  0.01156669 ...,  0.041296   -0.02286952\n",
      "    -0.14389446]]\n",
      "\n",
      "  [[ 0.09224495  0.01622674 -0.06678429 ...,  0.0276334  -0.07222521\n",
      "     0.06384867]\n",
      "   [ 0.00606321  0.05197761  0.03456064 ..., -0.01010545 -0.15198211\n",
      "     0.09631934]\n",
      "   [ 0.00776612 -0.08246826  0.06032583 ..., -0.05764055  0.06105212\n",
      "    -0.00581952]]\n",
      "\n",
      "  [[-0.05993195  0.01222826  0.03296838 ...,  0.03130878  0.03751753\n",
      "    -0.024904  ]\n",
      "   [ 0.01534607 -0.01715485 -0.08379588 ...,  0.15480976 -0.1168099\n",
      "    -0.06874805]\n",
      "   [-0.12155374 -0.09631383 -0.07001221 ...,  0.16224614 -0.00347467\n",
      "    -0.03507717]]]\n",
      "\n",
      "\n",
      " [[[-0.02507116 -0.10870472 -0.13082139 ..., -0.17167495  0.06762534\n",
      "     0.1502248 ]\n",
      "   [-0.02714132 -0.10622574  0.04751731 ..., -0.11622041  0.11408148\n",
      "    -0.12311635]\n",
      "   [-0.01530214  0.01907106 -0.12855424 ..., -0.16844881  0.06416614\n",
      "    -0.06367827]]\n",
      "\n",
      "  [[ 0.09335198 -0.00479288  0.02213567 ..., -0.06588044 -0.01094866\n",
      "     0.09513061]\n",
      "   [ 0.04959439  0.00228199 -0.09055059 ...,  0.14907522  0.02252195\n",
      "     0.17274834]\n",
      "   [ 0.08101598  0.03498532 -0.043636   ..., -0.04032679  0.17703125\n",
      "    -0.07456809]]\n",
      "\n",
      "  [[-0.18650021  0.07814053  0.07949853 ..., -0.13802749  0.01679799\n",
      "    -0.00355719]\n",
      "   [ 0.1323906  -0.0950759  -0.07040253 ..., -0.02007877  0.03991398\n",
      "     0.05700162]\n",
      "   [ 0.08290287  0.13874625  0.04672506 ..., -0.02867217 -0.04690486\n",
      "    -0.03386902]]\n",
      "\n",
      "  [[-0.00986507 -0.00735654  0.05217924 ...,  0.02351238 -0.16810969\n",
      "     0.0468756 ]\n",
      "   [-0.01612451 -0.01887821 -0.05202255 ..., -0.05500069 -0.07377695\n",
      "    -0.11398219]\n",
      "   [ 0.08066579  0.02065529 -0.06983637 ...,  0.10331509 -0.03871667\n",
      "    -0.01458761]]\n",
      "\n",
      "  [[ 0.1857733  -0.1015592   0.05962673 ..., -0.11800704 -0.04247097\n",
      "     0.10349105]\n",
      "   [ 0.05706574  0.13637482 -0.03151322 ...,  0.08109368  0.06130136\n",
      "     0.02092161]\n",
      "   [-0.08369505 -0.07459023 -0.05909295 ..., -0.09138941 -0.13232163\n",
      "     0.08155868]]]\n",
      "\n",
      "\n",
      " [[[ 0.05223009 -0.05617794 -0.16077006 ...,  0.00665022  0.06255458\n",
      "     0.10788343]\n",
      "   [ 0.08950833  0.12613061  0.07420417 ..., -0.10928037  0.02796033\n",
      "     0.07996727]\n",
      "   [-0.06923559 -0.02311405 -0.0066545  ..., -0.02691278 -0.15588625\n",
      "    -0.01212545]]\n",
      "\n",
      "  [[ 0.01056605  0.12971407  0.13714223 ..., -0.03534864 -0.12514932\n",
      "    -0.16199405]\n",
      "   [ 0.02014007  0.11768488  0.13122277 ..., -0.17187729  0.0140139\n",
      "    -0.04691496]\n",
      "   [-0.00267713  0.01466085 -0.09722573 ..., -0.08921598 -0.00110881\n",
      "    -0.0454805 ]]\n",
      "\n",
      "  [[-0.11098935 -0.14009793 -0.03063524 ..., -0.005068    0.03478072\n",
      "     0.0803323 ]\n",
      "   [-0.06884564 -0.0487962  -0.02393085 ...,  0.14755626 -0.03700639\n",
      "    -0.12590913]\n",
      "   [-0.10496327  0.01565567  0.09481482 ...,  0.02025781 -0.02636343\n",
      "    -0.12023323]]\n",
      "\n",
      "  [[ 0.14873736 -0.06162508  0.06976606 ...,  0.05184479 -0.03313804\n",
      "    -0.02634036]\n",
      "   [ 0.03717185  0.0675271   0.15191676 ...,  0.0210666  -0.02449561\n",
      "    -0.139277  ]\n",
      "   [ 0.03581068  0.01309618  0.19045664 ..., -0.15655085 -0.14886592\n",
      "     0.10421944]]\n",
      "\n",
      "  [[ 0.10495595  0.11802106  0.06444889 ..., -0.08959889  0.0432586\n",
      "    -0.058305  ]\n",
      "   [-0.07093336 -0.01584711  0.05030471 ..., -0.01726904  0.16115616\n",
      "     0.11454102]\n",
      "   [-0.050744   -0.12898919  0.04873359 ..., -0.1027717  -0.07810881\n",
      "    -0.04161097]]]\n",
      "\n",
      "\n",
      " [[[-0.00593628  0.00419261 -0.02079924 ..., -0.03510244  0.05893018\n",
      "     0.06492653]\n",
      "   [-0.14284933 -0.08093548  0.0162618  ..., -0.04789748 -0.0695299\n",
      "     0.12879567]\n",
      "   [ 0.13818651 -0.1544632   0.00449269 ..., -0.07387715 -0.02276297\n",
      "    -0.1160702 ]]\n",
      "\n",
      "  [[-0.1704223   0.01694544  0.14851184 ..., -0.08189876  0.04497364\n",
      "    -0.12771161]\n",
      "   [-0.03381432 -0.1574727   0.18366343 ...,  0.0285823  -0.03653543\n",
      "    -0.14390296]\n",
      "   [ 0.13469912 -0.03471756 -0.07414754 ...,  0.10312257  0.01348385\n",
      "     0.03292139]]\n",
      "\n",
      "  [[ 0.07193578  0.07324421 -0.1259298  ...,  0.03354873  0.06075791\n",
      "    -0.07266003]\n",
      "   [ 0.04530961 -0.01236912 -0.13939761 ...,  0.138633    0.06833181\n",
      "    -0.03879663]\n",
      "   [ 0.16606791  0.01326527  0.09021451 ..., -0.00897733 -0.1132465\n",
      "    -0.18389311]]\n",
      "\n",
      "  [[-0.03951256 -0.0605026  -0.06966393 ..., -0.0463929  -0.10927041\n",
      "    -0.00683731]\n",
      "   [ 0.0377125   0.01448696 -0.16881555 ..., -0.18066852 -0.1008145\n",
      "     0.00481235]\n",
      "   [-0.04288668 -0.07590524  0.09091684 ...,  0.01036892 -0.04319569\n",
      "    -0.073253  ]]\n",
      "\n",
      "  [[-0.09032097 -0.01743832 -0.0869377  ..., -0.06622799  0.08500978\n",
      "     0.02940061]\n",
      "   [ 0.03552309 -0.03537515  0.06376549 ...,  0.11220165  0.0132148\n",
      "    -0.10723597]\n",
      "   [-0.04053522  0.00085747  0.04058335 ...,  0.15657917  0.14834386\n",
      "    -0.0218953 ]]]\n",
      "\n",
      "\n",
      " [[[ 0.19973332 -0.07261434 -0.10753415 ..., -0.01237449  0.02768598\n",
      "    -0.07925578]\n",
      "   [-0.11175773 -0.07831412 -0.09052713 ...,  0.0087121   0.04891082\n",
      "    -0.10578507]\n",
      "   [ 0.19051589  0.03820269  0.08352969 ...,  0.08535905 -0.12162451\n",
      "    -0.09403785]]\n",
      "\n",
      "  [[-0.0797546   0.09157541  0.12092128 ...,  0.06048781  0.04794059\n",
      "    -0.02672653]\n",
      "   [-0.15174067  0.00343234 -0.04262105 ..., -0.04453593  0.05412078\n",
      "     0.01551288]\n",
      "   [ 0.05730721  0.19129972  0.17136298 ...,  0.06026939  0.01784454\n",
      "     0.10715091]]\n",
      "\n",
      "  [[-0.16008323  0.09644582 -0.12344394 ...,  0.14696735  0.07396205\n",
      "    -0.05465374]\n",
      "   [ 0.08895724  0.04684636 -0.01754807 ..., -0.07287272 -0.01095547\n",
      "    -0.10868981]\n",
      "   [ 0.04633987 -0.06457528 -0.01404413 ...,  0.032284    0.06306554\n",
      "     0.09901734]]\n",
      "\n",
      "  [[-0.0887095  -0.04899822 -0.01127402 ..., -0.02730491 -0.14649364\n",
      "     0.0936428 ]\n",
      "   [-0.08527559 -0.0311989  -0.08623428 ...,  0.06849999 -0.01662476\n",
      "    -0.06834855]\n",
      "   [-0.07093776 -0.00065265 -0.06346827 ..., -0.00914978  0.12384092\n",
      "     0.17941537]]\n",
      "\n",
      "  [[ 0.11640719  0.15937869 -0.07743965 ...,  0.04543127 -0.0903609\n",
      "    -0.15247737]\n",
      "   [ 0.16321844 -0.15560071 -0.07213389 ...,  0.04484996 -0.1619896\n",
      "    -0.08497944]\n",
      "   [-0.02687841  0.0043685  -0.00450939 ...,  0.08922135  0.04144232\n",
      "     0.13657539]]]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(biases))\n",
    "    print(sess.run(filter_weight))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LENET-5模型\n",
    "\n",
    "```python\n",
    "input: [width=32, height=32, color=1]\n",
    "\n",
    "conv1:\n",
    "filter: [width=5, height=5, input-dims=1, output-dims=6]\n",
    "stride: [x-stride=1, y-stride=1]\n",
    "output: [width=(32-5+1)/1=28, height=(32-5+1)/1=28, output-dims=6]\n",
    "\n",
    "pool1:\n",
    "filter: [width=2, height=2]\n",
    "stride: [x-stride=2, y-stride=2]\n",
    "output: [width=28/2=14, height=28/2=14, output-dims=6]\n",
    "\n",
    "conv2:\n",
    "filter: [width=5, height=5, input-dims=6, output-dims=16]\n",
    "stride: [x-stride=1, y-stride=1]\n",
    "output: [width=(14-5+1)/1=10, height=(14-5+1)/1=10, output-dims=16]\n",
    "\n",
    "pool2:\n",
    "filter: [width=2, height=2]\n",
    "stride: [x-stride=2, y-stride=2]\n",
    "output: [width=10/2=5, height=10/2=5, output-dims=16]\n",
    "\n",
    "full-connect1:\n",
    "w: [5*5*16=400, 120]\n",
    "output: [120]\n",
    "\n",
    "full-connect2:\n",
    "w: [120, 84]\n",
    "output: [84]\n",
    "\n",
    "full-connect3:\n",
    "w: [84, 10]\n",
    "output: [10]\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
